from concurrent.futures import ThreadPoolExecutor
import requests
import pymysql
import time

class TxzpSpider:
    url = 'https://careers.tencent.com/tencentcareer/api/post/Query?timestamp=1728007701507&countryId=&cityId=&bgIds=&productId=&categoryId=&parentCategoryId=&attrId=&keyword=java&pageIndex={page}&pageSize=10&language=zh-cn&area=cn'

    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.0 (KHTML, like Gecko) Chrome/98.0.4758.102 Safari/537.0'
    }

    def __init__(self):
        self.db = pymysql.connect(host='localhost', port=3306, user='root', passwd='888888', db='test', cursorclass=pymysql.cursors.DictCursor)  # 使用DictCursor方便处理数据
        self.cursor = self.db.cursor()

    @classmethod
    def get_info(cls, page):
        try:
            resp = requests.get(url=cls.url.format(page=page), headers=cls.headers, timeout=10)
            resp.raise_for_status()  # 如果请求失败，将抛出异常
            data = resp.json()['Data']['Posts']
            return data
        except requests.RequestException as e:
            print(f"请求第{page}页失败：{e}")
            return []

    def create_table(self):
        sql = """
            create table if not exists tx_works4(
                id int primary key auto_increment,
                work_name varchar(100) not null,
                country varchar(50),
                city varchar(50),
                work_desc text not null
            );
        """
        try:
            self.cursor.execute(sql)
            self.db.commit()
            print('表创建成功')
        except Exception as e:
            self.db.rollback()
            print(e, "创建失败")

    def insert_data(self, data_list):
        sql = """
            insert into tx_works4(
                work_name,
                country,
                city,
                work_desc
            )values (%s, %s, %s, %s);
        """
        try:
            self.cursor.executemany(sql, data_list)
            self.db.commit()
            print(f"成功插入{len(data_list)}条数据")
        except Exception as e:
            self.db.rollback()
            print(e, "插入数据失败")

    def main(self):
        self.create_table()
        data_list = []
        with ThreadPoolExecutor(max_workers=10) as executor:
            futures = [executor.submit(self.get_info, page) for page in range(1, 18)]
            for future in futures:
                data = future.result()
                data_list.extend(data)  # 收集所有页面的数据

        # 批量插入数据
        data_to_insert = [(w['RecruitPostName'], w['CountryName'], w['LocationName'], w['Responsibility']) for w in data_list]
        self.insert_data(data_to_insert)
        self.db.close()

if __name__ == '__main__':
    start = time.time()
    t = TxzpSpider()
    t.main()
    print('使用时间:', time.time() - start) # 1.43s